#encoding=utf8
import ply.lex as lex

tokenSet=set(['ABS','ALTER','AND','ARG',
'ASC','BY','COPY','COUNT','CREATE',
'DELETE','DESC','DROP','FROM','IN',
'INSERT','INT','INTERGER','INTO',
'IS','KEY','LENGTH','LIKE','LOAD',
'MAX','MIN','NOT','OR','ORDER',
'PRIMARY','REAL','REGEXP','RENAME',
'SAVE','SELECT','SHOW','SUM','TABLE','TABLES',
'TO','TYPEOF','VALUES','VARCHAR','WHERE'])

tokens = [
    'ID',  #表名|列名..
    'NUMBER', 
    'STRING', #TEXT???
    'FLOAT',
    'O_PER', 
    'O_RIGHT',
    'O_LEFT',
    'O_MUL',
    'O_ADD',
    'O_COMMA',
    'O_SUB',
    'O_EQUAL',
    'O_GT',
    'O_LT',
    'O_GTE',
    'O_LTE',
]+list(tokenSet)

def t_ID(t):
    r'[a-zA-Z_][#a-zA-Z_0-9]*'
    if t.value.upper() in tokenSet:
        t.type = t.value.upper()
        return t
    if t.value[0:4]=="TEXT":
        t.type = "STRING"
        t.value = t.value[4:]
        return t
    return t
    # print "Illegal word '%s'" % t.value


t_O_PER = r'\,'
t_O_RIGHT = r'\)'
t_O_LEFT = r'\('
t_O_MUL = r'\*'
t_O_ADD = r'\+'
t_O_COMMA = r'\,'
t_O_SUB = r'\-'
t_O_EQUAL = r'\='
t_O_GT = r'\>'
t_O_LT = r'\<'
t_O_GTE = r'\>\='
t_O_LTE = r'\<\='


def t_FLOAT(t):
    r'((\d*\.\d+)(E[\+-]?\d+)?|([1-9]\d*E[\+-]?\d+))'
    t.value = float(t.value)
    return t

def t_NUMBER(t):
    r'\d+'
    t.value = int(t.value)    
    return t

def t_newline(t):
    r'\n+'
    t.lexer.lineno += len(t.value)

t_ignore  = '; \t\n'

def t_error(t):
    print "Illegal character '%s'" % t.value[0]
    t.lexer.skip(1)



# 把sql语句中的字符串转换为TEXT123之类的形式. 实现方式 状态机
# 输入 sql 语句  
# 返回 处理后的语句和字符串映射表
def findCString(s,debug=False):
    s+="\n"
    strList = []
    state = "normal"
    stringNow = ""
    stringNew = ""
    for i,c in enumerate(s):
        if debug : print i,c,state,[stringNow] 
        if state =="start":
            stringNow=""
            if c=='\\': state = "trun"
            elif c=='"': state = 'end'
            else :
                state = "input"
                stringNow=str(c)
        elif state =="trun":
            if c=='"': stringNow+='"'
            elif c=='\\' : stringNow+='\\'
            elif c=='n' : stringNow+='\n'
            elif c=='r' : stringNow+='\r'
            elif c=='t' : stringNow+='\t'
            else : print "error escape #",i,c
            state = 'input'
        elif state =="input":
            if c=='\\':state = 'trun'
            elif c=='"':state = 'end'
            else :stringNow+=c
        elif state =="end":
            # print [stringNow]
            strList.append(stringNow)
            stringNew+= " TEXT%d "%(len(strList)-1)
            stringNew+=str(c)
            state = 'normal'
        elif state == 'normal':
            if c=='"' :state='start'
            else :stringNew +=str(c)
            continue;
    return stringNew,strList

lexer = lex.lex(debug=0)

def test(t):
    s=findCString(t)
    print s[0]
    lexer.input(s[0])
    for i in lexer :print i

def testAll():
    for line in open("sqlTest.py").readlines():
        test(line)
    

if __name__ == '__main__':
    testAll()
